package com.yufei.infoExtractor.parallel.thread;

import it.sauronsoftware.cron4j.Task;
import it.sauronsoftware.cron4j.TaskExecutionContext;

import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.yufei.entity.Relatedlink;
import com.yufei.infoExtractor.context.HActionContext;
import com.yufei.infoExtractor.extractor.fetchdecision.DecisionMaker;
import com.yufei.infoExtractor.pfw.InfoExtractorDao;
import com.yufei.infoExtractor.pfw.entity.CatalogMapping;
import com.yufei.infoExtractor.pfw.entity.PropertyMatch;
import com.yufei.infoExtractor.pfw.entity.Seedsite;
import com.yufei.infoExtractor.task.InfoExtractorCommonTask;
import com.yufei.infoExtractor.util.AppUtil;
import com.yufei.infoExtractor.util.FactoryUtil;
import com.yufei.utils.CommonUtil;
import com.yufei.utils.EncryptUtil;
import com.yufei.utils.ExceptionUtil;
import com.yufei.utils.HtmlUtil;
import com.yufei.utils.PatternUtils;

/**
 * @author jasstion
   2012-11-23
   url extractor
 *
 */
public class UrlParserThread extends Task {
	public static final Log mLog = LogFactory.getLog(UrlParserThread.class);
	public InfoExtractorDao infoExtractorDao=(InfoExtractorDao) AppUtil.getBeanFromBeanContainer(InfoExtractorDao.class);
    private HActionContext context=null;
	private List<String> readyUrls = null;
	private CountDownLatch countDownLatch = null;
    
	public UrlParserThread(ThreadLocal<HActionContext> threadLocalcontext, List<String> readyUrls,
			CountDownLatch countDownLatch) {
		super();
    this.context=threadLocalcontext.get();
    this.readyUrls = readyUrls;
		this.countDownLatch = countDownLatch;
	}

	

	public List<String> getReadyUrls() {
		return readyUrls;
	}

	public void setReadyUrls(List<String> readyUrls) {
		this.readyUrls = readyUrls;
	}

	public CountDownLatch getCountDownLatch() {
		return countDownLatch;
	}

	public void setCountDownLatch(CountDownLatch countDownLatch) {
		this.countDownLatch = countDownLatch;
	}

	
	
		
	public void extractorUrls(HActionContext context,TaskExecutionContext taskExecutionContext){
		// TODO Auto-generated method stub
	//	InfoExtractorCache< String, String> urlCache=(InfoExtractorCache<String, String>) context.get("urlCache");
		Integer currentDepth = context.getCurrentDepth();
		Integer depth = context.getSeedsite().getDepth();
		Set[] linkSet =context.getLinkSets();
		Iterator<String> iter = readyUrls.iterator();
		String htmlContent = null;
		List<PropertyMatch> propertyMatches=context.getSeedsite().getPattern().getPropertyMatches();
		Seedsite seed=context.getSeedsite();
		String  url1 = null;
		//UrlExtractor urlExtractor=(UrlExtractor) context.get(UrlExtractor.EXTRACTOR_KEY);
        //对采集到的链接进行初步处理：递归抓取，获取每个链接的内容以及保存relatedlink
		for(String url:readyUrls){
			
			taskExecutionContext.pauseIfRequested();
			 if(taskExecutionContext.isStopped()){
	        	   throw new IllegalStateException();
	           }
            
			 htmlContent=HtmlUtil.getHtmlContent(url, 2, 0,seed.getDataRetrieverFeatures().getRequestExecuteJs());
			if(CommonUtil.isEmptyOrNull(htmlContent)){
				continue;
			}
	
			String filterregexes = (String) PropertyMatch.getPropertyMatchFromListByPropertyName("linkFilterRegex", propertyMatches).getMatch().get(PropertyMatch.regex_key);
				
			List<String> filterUrls = PatternUtils.getListStrByRegex(
					htmlContent, filterregexes);
			if (CommonUtil.isEmptyOrNull(filterUrls)) {
				mLog.debug("在对链接：'"+url+"'进行子链接提取的过程中，根据链接匹配正则表达式'"+filterregexes+"'对网页内容：'"+htmlContent+"'匹配失败，请检查网页内容或者是链接匹配表达式是否有误");
				continue;
			}
			int totalValidateUrlCount=0;
			for (String extractedUrl : filterUrls) {
			      
				try {
				mLog.debug("开始对子链接:'"+extractedUrl+"'进行链接合法性检测以及链接补全");
				taskExecutionContext.pauseIfRequested();
				 if(taskExecutionContext.isStopped()){
		        	   throw new IllegalStateException();
		           }
				if (!CommonUtil.isValidUrl(extractedUrl)) {
	
					continue;
	
				}
				url1 = CommonUtil.formatUrl(extractedUrl);
				url1 = CommonUtil.normalizeUrl(extractedUrl, url);
				
		      if(url1==null){
		    	  continue;
		      }
		      
				boolean isOk=FactoryUtil.getDecisionMakerByType(DecisionMaker.hasNotFetched).makeDecision(context, url1);
				if (!isOk) {
					
					continue;
				}
				mLog.debug("补全后的链接为'"+url1+"'");
				totalValidateUrlCount++;
			

				context.getInfoExtractorCommonTask().increaseUrlExtractedNumber();
				//如果是最后一次递归解析链接的话，并且链接是目标链接则应该直接保存
				if(FactoryUtil.getDecisionMakerByType(DecisionMaker.isTargetLink).makeDecision(context, url1)){
					
						Class entityClass =context.getTargetEntity().getClass();
						Relatedlink tempLink=null;
						String tempHtmlContent=null;
						tempHtmlContent=HtmlUtil.getHtmlContent(url1, 2, 0,seed.getDataRetrieverFeatures().getRequestExecuteJs());
						if(tempHtmlContent==null){
							continue;
						}
						String contentHashCode=EncryptUtil.md5(tempHtmlContent);
							tempLink=(Relatedlink) entityClass.newInstance();
						
						//如果有必要设置此链接的分类关键字
							String keyWords=getKeyWords(url);
							//tempLink.setKeyWords(keyWords);
							   Relatedlink existedLink=(Relatedlink) infoExtractorDao.queryEntity("contentHashCode", contentHashCode, context.getTargetEntity().getClass());

							if(existedLink==null){
								saveRelatedLink(context.getInfoExtractorCommonTask(), tempHtmlContent, contentHashCode, url1,
										tempLink);
							}
							//更行
							else{
								updateRelatedLink(context.getInfoExtractorCommonTask(), tempHtmlContent, contentHashCode, url1, existedLink);
							}
					
						
						
					
					
				}
				
		
				
				if(!(currentDepth+2>=depth)){
					linkSet[currentDepth+1].add(url1);
				}
				
				

				//加入任务中的readyUrls中
				//urlExtractor.getTask().getReadyUrls().add(url1);
				} catch (Exception e1) {
					// TODO Auto-generated catch block
					mLog.info(ExceptionUtil.getExceptionDetailsMessage(e1));
					continue;
				} 
				
	
			}


			
			mLog.debug("子链接'"+url+"'总共抽取到有效链接个数为：'"+totalValidateUrlCount+"'");


		}
	}



	private void updateRelatedLink(
			InfoExtractorCommonTask infoExtractorCommonTask,
			String htmlContent, String contentHashCode, String url,
			Relatedlink existedLink) {
		// TODO Auto-generated method stub
		//更行时间
		existedLink.setUpdateTime(new Date());
		existedLink.setProcessed(false);
		existedLink.setOriginalHtmlContent(htmlContent);
		existedLink.setContentHashCode(contentHashCode);

		infoExtractorDao.saveEntity(existedLink);
		
		
	}



	private String getKeyWords(String url) {
		// TODO Auto-generated method stub
		String keyWords=null;
		List<CatalogMapping> catalogMappings=infoExtractorDao.queryListEntity(null, CatalogMapping.class);
		for(CatalogMapping ca:catalogMappings){
			if(url.contains(ca.getCatalogId())){
				keyWords=ca.getKeyWords();
				break;
			}
			
			
		}
		return keyWords;
	}



	private void saveRelatedLink(
			InfoExtractorCommonTask infoExtractorCommonTask,
			String htmlContent, String contentHashCode, String url,
			Relatedlink currentLink) {
		currentLink.setLink(url);
		currentLink.setFindTime(new Date());
		currentLink.setOriginalHtmlContent(htmlContent);
		//用于表示当前的采集信息属于那个task
		currentLink.setTaskId(infoExtractorCommonTask.getTask().getId());
		currentLink.setProcessed(false);
		currentLink.setContentHashCode(contentHashCode);
		infoExtractorDao.saveEntity(currentLink);

	}





	@Override
	public void execute(TaskExecutionContext taskExecutionContext) throws RuntimeException {
		// TODO Auto-generated method stub
		extractorUrls(context,taskExecutionContext);

		countDownLatch.countDown();
		mLog.info("线程执行完毕");

	}
	public boolean canBePaused() {
		return true;
	}

	public boolean canBeStopped() {
		return true;
	}

	public boolean supportsCompletenessTracking() {
		return true;
	}

	public boolean supportsStatusTracking() {
		return true;
	}


}
